#!/usr/bin/env python
'''
Created on 2/12/15

@author: Rich Plevin (rich@plevin.com)

Simplified interface for running batch queries against GCAM's XML database to
generate CSV files. Generates an XML batch query file based on command-line
arguments and a configuration file which stores information about the user's
GCAM setup, e.g., the location of the folder containing the Main_User_Workspace.

This functionality (and more) is available in the gcammcs framework, but the
advantage of this script is that it requires only standard Python modules.
'''

# Copyright (c) 2015, Richard Plevin.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the
#    distribution.
#
# 3. Neither the name of the copyright holder nor the names of its
#    contributors may be used to endorse or promote products derived
#    from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import os
import sys
import argparse
import string
from common import readConfigFiles, getParam, getParamAsBoolean
from Xvfb import Xvfb

PROGRAM = os.path.basename(__file__)
VERSION = "0.5"


def parseArgs():
    parser = argparse.ArgumentParser(
        prog=PROGRAM,
        fromfile_prefix_chars='@',      # use "@" before value to substitute contents of file as arguments
        description='''
    Run one or more GCAM database queries by generating and running the
    required XML batch query files. The results are placed in a file in the
    specified output directory with a name composed of the basename of the
    XML query file plus the scenario name. For example,
    "batchQuery.py -o. -s MyReference,MyPolicyCase liquids-by-region.xml"
    would generate query results into the files ./liquids-by-region-MyReference.csv
    and ./liquids-by-region-MyPolicyCase.csv. This script populates an initial
    configuration file in ~/.gcam.cfg when first run. The config file should be
    customized as needed, e.g., to set "GcamRoot" to the directory holding your
    Main_User_Workspace unless it happens to live in ~/GCAM, which is the default value.''')

    # REQUIRED
    parser.add_argument('queryFile', type=str, nargs='*',
                        help='''A file or files, each holding an XML query to run. (The ".xml" suffix will be added if needed.)
                                If an argument is preceded by the "@" sign, it is read and its contents substituted as the
                                values for this argument. That means you can store queries to run in a file (one per line) and
                                just reference the file by preceding the filename argument with "@".''')

    parser.add_argument('-b', '--batchFile', type=str,
                        help='The XML batch query file to generate (default taken from config parameter "BatchFile")')

    parser.add_argument('-d', '--xmldb', type=str,
                         help='''The XML database to query (default is 'output/database.dbxml' in the GCAM.Workspace.
                                 Overrides the -w flag.''')

    parser.add_argument('-n', '--noRun', action="store_true",
                        help="Show the command to be run, but don't run it")

    parser.add_argument('-o', '--outputDir', type=str,
                         help='Where to output the result (default taken from config parameter "OutputDir")')

    parser.add_argument('-q', '--queryDir', type=str, default=None,
                        help='The directory to look in to find query files. Defaults to value of config parameter GCAM.QueryDir')

    parser.add_argument('-s', '--scenario', type=str, default='Reference',
                        help='''A comma-separated list of scenarios to run the query/queries for (default is "Reference")
                                Note that these refer to a scenarios in database.dbxml.''')

    parser.add_argument('-v', '--verbose', action='count',
                        help="Show command being executed.")

    parser.add_argument('-V', '--version', action='version', version='%(prog)s ' + VERSION)

    parser.add_argument('-w', '--workspace', type=str, default='',
                        help='''The workspace directory in which to find output/database.dbxml.
                                Defaults to value of config file parameter GCAM.Workspace.
                                Overridden by the -d flag.''')

    args = parser.parse_args()

    return args

BatchQueryTemplate = string.Template("""<?xml version="1.0"?>
<!-- WARNING: this file is automatically generated. Changes may be overwritten. -->
<ModelInterfaceBatch>
    <class name="ModelInterface.ModelGUI2.DbViewer">
        <command name="XMLDB Batch File">
            <scenario name="${scenario}"/>
            <queryFile>${queryFile}</queryFile>
            <outFile>${csvFile}</outFile>
            <xmldbLocation>${xmldb}</xmldbLocation>
            <batchQueryResultsInDifferentSheets>false</batchQueryResultsInDifferentSheets>
            <batchQueryIncludeCharts>false</batchQueryIncludeCharts>
            <batchQuerySplitRunsInDifferentSheets>false</batchQuerySplitRunsInDifferentSheets>
            <batchQueryReplaceResults>true</batchQueryReplaceResults>
        </command>
    </class>
</ModelInterfaceBatch>
""")


def main():
    args = parseArgs()
    readConfigFiles()

    jarFile     = getParam('GCAM.JarFile')
    javaLibPath = getParam('GCAM.JavaLibPath')
    javaArgs    = getParam('GCAM.JavaArgs')
    batchFile   = args.batchFile or getParam('GCAM.BatchFile')
    outputDir   = args.outputDir or getParam('GCAM.OutputDir')
    workspace   = args.workspace or getParam('GCAM.Workspace')
    xmldb       = args.xmldb     or os.path.join(workspace, 'output/database.dbxml')
    queryDir    = args.queryDir  or getParam('GCAM.QueryDir')
    scenarios   = args.scenario.split(',')
    queryFiles  = args.queryFile

    if not queryFiles:
        print "Error: At least one query file must be specified"
        sys.exit(1)

    # TBD: test that outputDir exists
    if not os.path.lexists(outputDir):
        print "Output directory '%s' does not exist" % outputDir
        sys.exit(1)

    for scenario in scenarios:
        for queryFile in queryFiles:
            basename = os.path.basename(queryFile)
            mainPart, extension = os.path.splitext(basename)

            if extension != '.xml':
                queryFile += '.xml'

            if not os.path.lexists(queryFile):
                # of not in the indicated location, look in the QueryDir
                queryFile = os.path.join(queryDir, queryFile)
                if not os.path.lexists(queryFile):
                    print "Error: query file '%s' was not found." % queryFile
                    sys.exit(1)

            csvFile = "%s-%s.csv" % (mainPart, scenario)
            csvPath = os.path.abspath(os.path.join(outputDir, csvFile))

            # Write a batch file for ModelInterface to invoke the query on the named scenario, and put output where specified
            # TBD: See if multiple queries can be written to one batch file, and if multiple <scenario> elements are allowed.
            # TBD: Could write out a file with <query>xxx</query> where xxx is filled with <aQuery>...</aQuery> elements.
            # TBD: This would suggest rewriting the query XML files to have just the <aQuery> elements in them.
            batchFileText = BatchQueryTemplate.substitute(scenario=scenario, queryFile=queryFile, csvFile=csvPath, xmldb=xmldb)
            with open(batchFile, "w") as fp:
                fp.write(batchFileText)

            if args.verbose:
                print "Generating %s" % csvPath

            command = "java %s -Djava.library.path='%s' -jar '%s' -b '%s'" % (javaArgs, javaLibPath, jarFile, batchFile)

            if args.verbose or args.noRun:
                print command

            if not args.noRun:

                useVirtualBuffer = getParamAsBoolean('GCAM.UseVirtualBuffer')

                try:
                    xvfb = None
                    if useVirtualBuffer:
                        xvfb = Xvfb()      # run the X virtual buffer to suppress popup windows

                    os.system(command)

                except:
                    raise

                finally:        # caller (runGCAM function) traps exceptions, so we just re-raise
                    if xvfb:
                        xvfb.terminate()

if __name__ == '__main__':
    main()
